{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import Input, Dense, Reshape, Flatten, Dropout\n",
    "from keras.layers import BatchNormalization, Activation, ZeroPadding2D\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from keras.optimizers import Adam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_discriminator():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(41, input_dim=41, activation='relu'))  # discriminator takes 41 values from our dataset\n",
    "    model.add(Dense(30, activation='relu'))\n",
    "    model.add(Dense(15, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))  # outputs 0 to 1, 1 being read and 0 being fake\n",
    "\n",
    "    # model.summary()\n",
    "\n",
    "    attack = Input(shape=(41,))\n",
    "    validity = model(attack)\n",
    "\n",
    "    return Model(attack, validity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_generator(hidden1, hidden2, hidden3):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(hidden1, input_dim=41))  # arbitrarily selected 100 for our input noise vector?\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(hidden2))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(hidden3))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(41, activation='relu'))  # outputs a generated vector of the same size as our data (41)\n",
    "\n",
    "    # model.summary()\n",
    "\n",
    "    noise = Input(shape=(41,))\n",
    "    attack = model(noise)\n",
    "    return Model(noise, attack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainGAN(gen_hidden1, gen_hidden2, gen_hidden3):\n",
    "    batch_size = 256\n",
    "    epochs = 7000\n",
    "    optimizer = Adam(0.0002, 0.5)\n",
    "    \n",
    "    dataframe = pd.read_csv('../../CSV/kdd_neptune_only_5000.csv').sample(500) # sample 100 data points randomly from the csv\n",
    "    \n",
    "    # apply \"le.fit_transform\" to every column (usually only works on 1 column)\n",
    "    le = LabelEncoder()\n",
    "    dataframe_encoded = dataframe.apply(le.fit_transform)\n",
    "    dataset = dataframe_encoded.values\n",
    "    \n",
    "    #to visually judge results\n",
    "    print(\"Real neptune attacks:\")\n",
    "    print(dataset[:2])\n",
    "    \n",
    "    # Set X as our input data and Y as our label\n",
    "    X_train = dataset[:, 0:41].astype(float)\n",
    "    Y_train = dataset[:, 41]\n",
    "    \n",
    "    # labels for data. 1 for valid attacks, 0 for fake (generated) attacks\n",
    "    valid = np.ones((batch_size, 1))\n",
    "    fake = np.zeros((batch_size, 1))\n",
    "    \n",
    "    # build the discriminator portion\n",
    "    discriminator = build_discriminator();\n",
    "    discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    # build the generator portion\n",
    "    generator = build_generator(gen_hidden1, gen_hidden2, gen_hidden3)\n",
    "    \n",
    "    #input and output of our combined model\n",
    "    z = Input(shape=(41,))\n",
    "    attack = generator(z)\n",
    "    validity = discriminator(attack)\n",
    "    \n",
    "    # build combined model from generator and discriminator\n",
    "    combined = Model(z, validity)\n",
    "    combined.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "    \n",
    "    #break condition for training (when diverging)\n",
    "    loss_increase_count = 0;\n",
    "    prev_g_loss = 0;\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "\n",
    "        # ---------------------\n",
    "        #  Train Discriminator\n",
    "        # ---------------------\n",
    "        \n",
    "        # selecting batch_size random attacks from our training data\n",
    "        idx = np.random.randint(0, X_train.shape[0], batch_size)\n",
    "        attacks = X_train[idx]\n",
    "        \n",
    "        # generate a matrix of noise vectors\n",
    "        noise = np.random.normal(0, 1, (batch_size, 41))\n",
    "        \n",
    "        # create an array of generated attacks\n",
    "        gen_attacks = generator.predict(noise)\n",
    "        \n",
    "        # loss functions, based on what metrics we specify at model compile time\n",
    "        d_loss_real = discriminator.train_on_batch(attacks, valid)\n",
    "        d_loss_fake = discriminator.train_on_batch(gen_attacks, fake)\n",
    "        d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)\n",
    "        \n",
    "        # generator loss function\n",
    "        g_loss = combined.train_on_batch(noise, valid)\n",
    "        \n",
    "        if epoch % 100 == 0:\n",
    "            print(\"%d [D loss: %f, acc.: %.2f%%] [G loss: %f] [Loss change: %.3f, Loss increases: %.0f]\" % (epoch, d_loss[0], 100 * d_loss[1], g_loss, g_loss - prev_g_loss, loss_increase_count))\n",
    "        \n",
    "        # if our generator loss icreased this iteration, increment the counter by 1\n",
    "        if (g_loss - prev_g_loss) > 0:\n",
    "            loss_increase_count = loss_increase_count + 1\n",
    "        else: \n",
    "            loss_increase_count = 0  # otherwise, reset it to 0, we are still training effectively\n",
    "            \n",
    "        prev_g_loss = g_loss\n",
    "            \n",
    "        if loss_increase_count > 5:\n",
    "            print('Stoping on iteration: ', epoch)\n",
    "            break\n",
    "            \n",
    "        if epoch % 20 == 0:\n",
    "            f = open(\"GANresultsNeptune.txt\", \"a\")\n",
    "            np.savetxt(\"GANresultsNeptune.txt\", gen_attacks, fmt=\"%.0f\")\n",
    "            f.close()\n",
    "            \n",
    "    # peek at our results\n",
    "    results = np.loadtxt(\"GANresultsNeptune.txt\")\n",
    "    print(\"Generated Neptune attacks: \")\n",
    "    print(results[:2])\n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['neptune.' 'normal.']\n",
      "[0]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Random Number Generator\n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "np.random.seed(seed)\n",
    "\n",
    "# load dataset\n",
    "\n",
    "dataframe = pd.read_csv(\"../../CSV/normalAndNeptune.csv\")#, header=True) \n",
    "\n",
    "# samples 10000 random data points from 500k\n",
    "dataframe = dataframe.sample(n=15000)\n",
    "# LabelEncoder, turns all our categorical data into integers\n",
    "le = LabelEncoder()\n",
    "\n",
    "# apply \"le.fit_transform\" to every column (usually only works on 1 column)\n",
    "dataframe_encoded = dataframe.apply(le.fit_transform)\n",
    "attack_labels = le.classes_\n",
    "indices_of_neptune = np.where(attack_labels == 'neptune.')\n",
    "neptune_index = indices_of_neptune[0]\n",
    "dataset = dataframe_encoded.values\n",
    "\n",
    "print(attack_labels)\n",
    "print(neptune_index)\n",
    "\n",
    "#Set X as our input data and Y as our label\n",
    "X = dataset[:,0:41].astype(float)\n",
    "Y = dataset[:,41]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "# encode class values as integers\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "encoded_Y = encoder.transform(Y)\n",
    "# convert integers to dummy variables (i.e. one hot encoded)\n",
    "dummy_y = np_utils.to_categorical(encoded_Y)\n",
    "# print(dummy_y)\n",
    "#print(len(dummy_y[0]))\n",
    "num_of_classes = len(dummy_y[0])  # the length of dummy y is the number of classes we have in our small sample\n",
    "# since we are randomly sampling from a large dataset, we might not get 1 of every class in our sample\n",
    "# we need to set output layer to be equal to the length of our dummy_y vectors\n",
    "print(num_of_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define baseline model\n",
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    \n",
    "    inputs = 41\n",
    "    hidden_layer1 = 10\n",
    "    hidden_layer2 = 5\n",
    "    hidden_layer3 = 0\n",
    "    outputs = num_of_classes  #needs to be this variable in case we forget to sample. Could end up having 10 classes or 12, etc\n",
    "    \n",
    "    model.add(Dense(hidden_layer1, input_dim=inputs, activation='relu'))\n",
    "    if hidden_layer2 != 0:\n",
    "        model.add(Dense(hidden_layer2, activation='relu'))\n",
    "    if hidden_layer3 != 0:\n",
    "        model.add(Dense(hidden_layer3, activation='relu'))\n",
    "    model.add(Dense(outputs, activation='softmax'))\n",
    "    \n",
    "    # Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) #optimizer=adam\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/32\n",
      " - 1s - loss: 6.5063 - acc: 0.5590\n",
      "Epoch 2/32\n",
      " - 0s - loss: 1.2563 - acc: 0.8716\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2356 - acc: 0.9817\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.1958 - acc: 0.9830\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0743 - acc: 0.9856\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0316 - acc: 0.9870\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0182 - acc: 0.9920\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0072 - acc: 0.9982\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0050 - acc: 0.9995\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0046 - acc: 0.9996\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0026 - acc: 0.9996\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0024 - acc: 0.9996\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9996\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9996\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9996\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9996\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9996\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9996\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9998\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9998\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 9.6465e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 9.3441e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 8.6916e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 1.3854 - acc: 0.8867\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.0332 - acc: 0.9927\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0149 - acc: 0.9967\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0124 - acc: 0.9973\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0107 - acc: 0.9979\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0090 - acc: 0.9981\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0075 - acc: 0.9982\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0060 - acc: 0.9986\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0050 - acc: 0.9989\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0040 - acc: 0.9990\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0031 - acc: 0.9992\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0025 - acc: 0.9994\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9995\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9996\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9996\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9997\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9997\n",
      "Epoch 20/32\n",
      " - 0s - loss: 9.7807e-04 - acc: 0.9998\n",
      "Epoch 21/32\n",
      " - 0s - loss: 9.2293e-04 - acc: 0.9998\n",
      "Epoch 22/32\n",
      " - 0s - loss: 8.5089e-04 - acc: 0.9999\n",
      "Epoch 23/32\n",
      " - 0s - loss: 7.7464e-04 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 7.0149e-04 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 6.3837e-04 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 5.6519e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 5.0267e-04 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 4.6596e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 3.9968e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 3.5219e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 3.0217e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 2.6026e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 6.3574 - acc: 0.5791\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.1725 - acc: 0.9590\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0927 - acc: 0.9741\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0679 - acc: 0.9833\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0503 - acc: 0.9903\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0392 - acc: 0.9913\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0275 - acc: 0.9917\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0186 - acc: 0.9919\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0136 - acc: 0.9921\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0111 - acc: 0.9964\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0094 - acc: 0.9971\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0087 - acc: 0.9974\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0081 - acc: 0.9976\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0078 - acc: 0.9973\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0074 - acc: 0.9976\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0071 - acc: 0.9975\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0068 - acc: 0.9979\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0065 - acc: 0.9979\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0063 - acc: 0.9979\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0060 - acc: 0.9980\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0057 - acc: 0.9981\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0055 - acc: 0.9984\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0053 - acc: 0.9986\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0050 - acc: 0.9988\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0049 - acc: 0.9989\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0047 - acc: 0.9990\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0045 - acc: 0.9989\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0040 - acc: 0.9990\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0027 - acc: 0.9993\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0024 - acc: 0.9993\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9993\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0021 - acc: 0.9993\n",
      "Epoch 1/32\n",
      " - 1s - loss: 2.0693 - acc: 0.8069\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.3520 - acc: 0.9876\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.3123 - acc: 0.9900\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2811 - acc: 0.9909\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.2543 - acc: 0.9931\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.2308 - acc: 0.9949\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.2104 - acc: 0.9965\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1921 - acc: 0.9982\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1760 - acc: 0.9990\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1616 - acc: 0.9998\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1488 - acc: 0.9999\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1374 - acc: 0.9999\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.1271 - acc: 0.9999\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.1179 - acc: 0.9999\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.1094 - acc: 0.9999\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.1018 - acc: 0.9999\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0948 - acc: 0.9999\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0885 - acc: 0.9999\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0827 - acc: 0.9999\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0774 - acc: 0.9999\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0726 - acc: 0.9999\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0681 - acc: 0.9999\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0640 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0603 - acc: 1.0000\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0568 - acc: 1.0000\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0536 - acc: 1.0000\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0506 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0478 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0453 - acc: 1.0000\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0429 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0407 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0386 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 4.4421 - acc: 0.7024\n",
      "Epoch 2/32\n",
      " - 0s - loss: 2.0430 - acc: 0.8712\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2516 - acc: 0.9977\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2206 - acc: 0.9991\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.1974 - acc: 0.9992\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.1778 - acc: 0.9992\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.1610 - acc: 0.9995\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1464 - acc: 0.9994\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1335 - acc: 0.9996\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1223 - acc: 0.9996\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1123 - acc: 0.9996\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1034 - acc: 0.9997\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0954 - acc: 0.9996\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0883 - acc: 0.9997\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0819 - acc: 0.9997\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0761 - acc: 0.9997\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0710 - acc: 0.9996\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0662 - acc: 0.9997\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0619 - acc: 0.9997\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0580 - acc: 0.9997\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0545 - acc: 0.9997\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0512 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0482 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0455 - acc: 0.9998\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0429 - acc: 0.9998\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0406 - acc: 0.9998\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0384 - acc: 0.9998\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0364 - acc: 0.9998\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0345 - acc: 0.9998\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0328 - acc: 0.9998\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0312 - acc: 0.9998\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0297 - acc: 0.9998\n",
      "Epoch 1/32\n",
      " - 1s - loss: 6.7366 - acc: 0.5693\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.8176 - acc: 0.9353\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0040 - acc: 0.9990\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0031 - acc: 0.9995\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0027 - acc: 0.9995\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0025 - acc: 0.9996\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9996\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9996\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9997\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9997\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9997\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9997\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9997\n",
      "Epoch 15/32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9998\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9998\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 21/32\n",
      " - 0s - loss: 9.9173e-04 - acc: 0.9998\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 9.2374e-04 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 9.0054e-04 - acc: 0.9998\n",
      "Epoch 25/32\n",
      " - 0s - loss: 8.4276e-04 - acc: 0.9998\n",
      "Epoch 26/32\n",
      " - 0s - loss: 8.3150e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 7.9061e-04 - acc: 0.9998\n",
      "Epoch 28/32\n",
      " - 0s - loss: 7.5055e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 7.2023e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 6.9713e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 6.6152e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 6.1333e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 0.4251 - acc: 0.9350\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.0608 - acc: 0.9941\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0091 - acc: 0.9985\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0026 - acc: 0.9995\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0021 - acc: 0.9996\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9996\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9997\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9997\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9998\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9998\n",
      "Epoch 13/32\n",
      " - 0s - loss: 9.7622e-04 - acc: 0.9998\n",
      "Epoch 14/32\n",
      " - 0s - loss: 9.1251e-04 - acc: 0.9999\n",
      "Epoch 15/32\n",
      " - 0s - loss: 8.7894e-04 - acc: 0.9998\n",
      "Epoch 16/32\n",
      " - 0s - loss: 8.1374e-04 - acc: 0.9999\n",
      "Epoch 17/32\n",
      " - 0s - loss: 8.0041e-04 - acc: 0.9999\n",
      "Epoch 18/32\n",
      " - 0s - loss: 7.5602e-04 - acc: 0.9999\n",
      "Epoch 19/32\n",
      " - 0s - loss: 7.3638e-04 - acc: 0.9999\n",
      "Epoch 20/32\n",
      " - 0s - loss: 6.9997e-04 - acc: 0.9999\n",
      "Epoch 21/32\n",
      " - 0s - loss: 6.7805e-04 - acc: 0.9999\n",
      "Epoch 22/32\n",
      " - 0s - loss: 6.4855e-04 - acc: 0.9999\n",
      "Epoch 23/32\n",
      " - 0s - loss: 6.2568e-04 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 6.0781e-04 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 5.7829e-04 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 5.6831e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 5.4836e-04 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 5.4135e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 5.1287e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 4.9117e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 4.6501e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 4.6393e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 0.4374 - acc: 0.8951\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.3504 - acc: 0.9716\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2944 - acc: 0.9964\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2640 - acc: 0.9982\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.2387 - acc: 0.9984\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.2166 - acc: 0.9988\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.1971 - acc: 0.9990\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1798 - acc: 0.9993\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1644 - acc: 0.9998\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1507 - acc: 0.9999\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1385 - acc: 0.9999\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1275 - acc: 0.9999\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.1176 - acc: 0.9999\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.1087 - acc: 1.0000\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.1007 - acc: 1.0000\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0934 - acc: 1.0000\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0869 - acc: 1.0000\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0810 - acc: 0.9999\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0756 - acc: 1.0000\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0707 - acc: 1.0000\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0662 - acc: 0.9999\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0621 - acc: 1.0000\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0583 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0548 - acc: 1.0000\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0517 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0487 - acc: 1.0000\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0460 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0435 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0411 - acc: 1.0000\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0389 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0369 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0350 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 4.2616 - acc: 0.6650\n",
      "Epoch 2/32\n",
      " - 0s - loss: 1.5041 - acc: 0.8938\n",
      "Epoch 3/32\n",
      " - 0s - loss: 1.1412 - acc: 0.9227\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2433 - acc: 0.9796\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0113 - acc: 0.9983\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0071 - acc: 0.9989\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0050 - acc: 0.9990\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0040 - acc: 0.9993\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0034 - acc: 0.9993\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0030 - acc: 0.9994\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0026 - acc: 0.9996\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0024 - acc: 0.9996\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0021 - acc: 0.9996\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9996\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9996\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9996\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9996\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9996\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9996\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9996\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9997\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9996\n",
      "Epoch 23/32\n",
      " - 0s - loss: 9.5698e-04 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 9.0068e-04 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 8.6603e-04 - acc: 0.9997\n",
      "Epoch 26/32\n",
      " - 0s - loss: 8.1538e-04 - acc: 0.9998\n",
      "Epoch 27/32\n",
      " - 0s - loss: 7.6188e-04 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 7.3086e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 7.0671e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 6.4216e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 6.2288e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 5.8848e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 3.7099 - acc: 0.7111\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.4463 - acc: 0.9696\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2995 - acc: 0.9868\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2632 - acc: 0.9963\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.2367 - acc: 0.9982\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.2148 - acc: 0.9985\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.1955 - acc: 0.9985\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1784 - acc: 0.9987\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1633 - acc: 0.9990\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1497 - acc: 0.9990\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1377 - acc: 0.9990\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1268 - acc: 0.9992\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.1172 - acc: 0.9993\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.1084 - acc: 0.9993\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.1006 - acc: 0.9994\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0934 - acc: 0.9995\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0870 - acc: 0.9996\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0811 - acc: 0.9997\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0758 - acc: 0.9998\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0709 - acc: 0.9998\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0664 - acc: 0.9998\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0624 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0586 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0551 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0519 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0490 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0462 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0437 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0413 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0391 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0371 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0352 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 3.7683 - acc: 0.6698\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.1691 - acc: 0.9849\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0383 - acc: 0.9952\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0295 - acc: 0.9967\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0274 - acc: 0.9965\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0255 - acc: 0.9971\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0230 - acc: 0.9970\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0209 - acc: 0.9973\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0181 - acc: 0.9977\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0157 - acc: 0.9978\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0132 - acc: 0.9976\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0109 - acc: 0.9983\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0092 - acc: 0.9981\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0079 - acc: 0.9987\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0068 - acc: 0.9988\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0057 - acc: 0.9987\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0050 - acc: 0.9990\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0043 - acc: 0.9990\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0040 - acc: 0.9991\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0033 - acc: 0.9992\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0037 - acc: 0.9990\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0029 - acc: 0.9995\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0030 - acc: 0.9995\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0032 - acc: 0.9993\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0021 - acc: 0.9996\n",
      "Epoch 27/32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 0s - loss: 0.0020 - acc: 0.9996\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9997\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9997\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9997\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 1/32\n",
      " - 1s - loss: 2.9310 - acc: 0.7453\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.1427 - acc: 0.9924\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0174 - acc: 0.9954\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0088 - acc: 0.9972\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0059 - acc: 0.9983\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0043 - acc: 0.9987\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0031 - acc: 0.9993\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9996\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9996\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9999\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9999\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 13/32\n",
      " - 0s - loss: 9.1586e-04 - acc: 0.9999\n",
      "Epoch 14/32\n",
      " - 0s - loss: 8.1121e-04 - acc: 0.9999\n",
      "Epoch 15/32\n",
      " - 0s - loss: 7.2522e-04 - acc: 0.9999\n",
      "Epoch 16/32\n",
      " - 0s - loss: 6.1989e-04 - acc: 1.0000\n",
      "Epoch 17/32\n",
      " - 0s - loss: 5.7714e-04 - acc: 1.0000\n",
      "Epoch 18/32\n",
      " - 0s - loss: 5.2813e-04 - acc: 1.0000\n",
      "Epoch 19/32\n",
      " - 0s - loss: 4.6848e-04 - acc: 1.0000\n",
      "Epoch 20/32\n",
      " - 0s - loss: 4.3790e-04 - acc: 0.9999\n",
      "Epoch 21/32\n",
      " - 0s - loss: 3.9630e-04 - acc: 1.0000\n",
      "Epoch 22/32\n",
      " - 0s - loss: 3.8776e-04 - acc: 1.0000\n",
      "Epoch 23/32\n",
      " - 0s - loss: 3.4045e-04 - acc: 1.0000\n",
      "Epoch 24/32\n",
      " - 0s - loss: 2.9824e-04 - acc: 1.0000\n",
      "Epoch 25/32\n",
      " - 0s - loss: 2.9253e-04 - acc: 1.0000\n",
      "Epoch 26/32\n",
      " - 0s - loss: 2.9026e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 2.4512e-04 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 2.3146e-04 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 2.2624e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 2.0293e-04 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 1.9111e-04 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 1.7336e-04 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 4.8020 - acc: 0.5846\n",
      "Epoch 2/32\n",
      " - 0s - loss: 1.4036 - acc: 0.8726\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2822 - acc: 0.9864\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2399 - acc: 0.9924\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.2139 - acc: 0.9937\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.1923 - acc: 0.9951\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.1743 - acc: 0.9963\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1583 - acc: 0.9971\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1447 - acc: 0.9975\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1321 - acc: 0.9979\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1211 - acc: 0.9981\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1111 - acc: 0.9983\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.1010 - acc: 0.9987\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0771 - acc: 0.9990\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0229 - acc: 0.9987\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0034 - acc: 0.9993\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0029 - acc: 0.9993\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0028 - acc: 0.9993\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9994\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9994\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9996\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 9.8190e-04 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 9.0612e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 7.0811e-04 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 6.6440e-04 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 6.4607e-04 - acc: 1.0000\n",
      "Epoch 30/32\n",
      " - 0s - loss: 5.5842e-04 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 5.0937e-04 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 4.8500e-04 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 0.1853 - acc: 0.9799\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.0146 - acc: 0.9973\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0086 - acc: 0.9983\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0062 - acc: 0.9990\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0045 - acc: 0.9992\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0035 - acc: 0.9991\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0027 - acc: 0.9994\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9995\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9996\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9996\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9996\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9997\n",
      "Epoch 13/32\n",
      " - 0s - loss: 9.2130e-04 - acc: 0.9997\n",
      "Epoch 14/32\n",
      " - 0s - loss: 6.4770e-04 - acc: 0.9998\n",
      "Epoch 15/32\n",
      " - 0s - loss: 5.2074e-04 - acc: 0.9999\n",
      "Epoch 16/32\n",
      " - 0s - loss: 4.5096e-04 - acc: 0.9999\n",
      "Epoch 17/32\n",
      " - 0s - loss: 4.0303e-04 - acc: 0.9999\n",
      "Epoch 18/32\n",
      " - 0s - loss: 3.5641e-04 - acc: 0.9999\n",
      "Epoch 19/32\n",
      " - 0s - loss: 3.1966e-04 - acc: 0.9999\n",
      "Epoch 20/32\n",
      " - 0s - loss: 2.9803e-04 - acc: 0.9999\n",
      "Epoch 21/32\n",
      " - 0s - loss: 2.6206e-04 - acc: 0.9999\n",
      "Epoch 22/32\n",
      " - 0s - loss: 2.4350e-04 - acc: 0.9999\n",
      "Epoch 23/32\n",
      " - 0s - loss: 2.2800e-04 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 2.1013e-04 - acc: 1.0000\n",
      "Epoch 25/32\n",
      " - 0s - loss: 1.9641e-04 - acc: 1.0000\n",
      "Epoch 26/32\n",
      " - 0s - loss: 1.8066e-04 - acc: 1.0000\n",
      "Epoch 27/32\n",
      " - 0s - loss: 1.6899e-04 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 1.5740e-04 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 1.4687e-04 - acc: 1.0000\n",
      "Epoch 30/32\n",
      " - 0s - loss: 1.3870e-04 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 1.3098e-04 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 1.2153e-04 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 0.1395 - acc: 0.9819\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.0085 - acc: 0.9979\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0043 - acc: 0.9988\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0032 - acc: 0.9995\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0028 - acc: 0.9996\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0024 - acc: 0.9996\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9997\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9998\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9998\n",
      "Epoch 10/32\n",
      " - 0s - loss: 9.6032e-04 - acc: 0.9999\n",
      "Epoch 11/32\n",
      " - 0s - loss: 7.7727e-04 - acc: 0.9999\n",
      "Epoch 12/32\n",
      " - 0s - loss: 5.2525e-04 - acc: 0.9999\n",
      "Epoch 13/32\n",
      " - 0s - loss: 3.6907e-04 - acc: 0.9999\n",
      "Epoch 14/32\n",
      " - 0s - loss: 2.1788e-04 - acc: 1.0000\n",
      "Epoch 15/32\n",
      " - 0s - loss: 1.6288e-04 - acc: 1.0000\n",
      "Epoch 16/32\n",
      " - 0s - loss: 1.2080e-04 - acc: 1.0000\n",
      "Epoch 17/32\n",
      " - 0s - loss: 1.0186e-04 - acc: 1.0000\n",
      "Epoch 18/32\n",
      " - 0s - loss: 9.1795e-05 - acc: 1.0000\n",
      "Epoch 19/32\n",
      " - 0s - loss: 7.9751e-05 - acc: 1.0000\n",
      "Epoch 20/32\n",
      " - 0s - loss: 6.9296e-05 - acc: 1.0000\n",
      "Epoch 21/32\n",
      " - 0s - loss: 6.2396e-05 - acc: 1.0000\n",
      "Epoch 22/32\n",
      " - 0s - loss: 5.7018e-05 - acc: 1.0000\n",
      "Epoch 23/32\n",
      " - 0s - loss: 5.2914e-05 - acc: 1.0000\n",
      "Epoch 24/32\n",
      " - 0s - loss: 4.8942e-05 - acc: 1.0000\n",
      "Epoch 25/32\n",
      " - 0s - loss: 4.4840e-05 - acc: 1.0000\n",
      "Epoch 26/32\n",
      " - 0s - loss: 4.1397e-05 - acc: 1.0000\n",
      "Epoch 27/32\n",
      " - 0s - loss: 3.8617e-05 - acc: 1.0000\n",
      "Epoch 28/32\n",
      " - 0s - loss: 3.5980e-05 - acc: 1.0000\n",
      "Epoch 29/32\n",
      " - 0s - loss: 3.4388e-05 - acc: 1.0000\n",
      "Epoch 30/32\n",
      " - 0s - loss: 3.4024e-05 - acc: 1.0000\n",
      "Epoch 31/32\n",
      " - 0s - loss: 3.1476e-05 - acc: 1.0000\n",
      "Epoch 32/32\n",
      " - 0s - loss: 2.9128e-05 - acc: 1.0000\n",
      "Epoch 1/32\n",
      " - 1s - loss: 8.4902 - acc: 0.4716\n",
      "Epoch 2/32\n",
      " - 0s - loss: 8.4627 - acc: 0.4750\n",
      "Epoch 3/32\n",
      " - 0s - loss: 8.4627 - acc: 0.4750\n",
      "Epoch 4/32\n",
      " - 0s - loss: 8.4627 - acc: 0.4750\n",
      "Epoch 5/32\n",
      " - 0s - loss: 8.4627 - acc: 0.4750\n",
      "Epoch 6/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 7/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 8/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 9/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 10/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 11/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 12/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 13/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 14/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 15/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 16/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 17/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 18/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 19/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 20/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 21/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 22/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 23/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 24/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 25/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 26/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 27/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 28/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 29/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 30/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 31/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 32/32\n",
      " - 0s - loss: 8.4626 - acc: 0.4750\n",
      "Epoch 1/32\n",
      " - 1s - loss: 3.0869 - acc: 0.7513\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.1010 - acc: 0.9957\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0527 - acc: 0.9984\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0200 - acc: 0.9990\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/32\n",
      " - 0s - loss: 0.0083 - acc: 0.9991\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0036 - acc: 0.9991\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0029 - acc: 0.9993\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0025 - acc: 0.9993\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9994\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9996\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9996\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9996\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9996\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9998\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9998\n",
      "Epoch 20/32\n",
      " - 0s - loss: 9.7702e-04 - acc: 0.9998\n",
      "Epoch 21/32\n",
      " - 0s - loss: 9.4960e-04 - acc: 0.9998\n",
      "Epoch 22/32\n",
      " - 0s - loss: 9.3540e-04 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 8.9780e-04 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 8.7828e-04 - acc: 0.9998\n",
      "Epoch 25/32\n",
      " - 0s - loss: 8.5892e-04 - acc: 0.9998\n",
      "Epoch 26/32\n",
      " - 0s - loss: 8.4468e-04 - acc: 0.9998\n",
      "Epoch 27/32\n",
      " - 0s - loss: 8.1485e-04 - acc: 0.9998\n",
      "Epoch 28/32\n",
      " - 0s - loss: 8.0916e-04 - acc: 0.9998\n",
      "Epoch 29/32\n",
      " - 0s - loss: 7.8707e-04 - acc: 0.9998\n",
      "Epoch 30/32\n",
      " - 0s - loss: 7.6303e-04 - acc: 0.9998\n",
      "Epoch 31/32\n",
      " - 0s - loss: 7.4670e-04 - acc: 0.9998\n",
      "Epoch 32/32\n",
      " - 0s - loss: 7.3008e-04 - acc: 0.9998\n",
      "Epoch 1/32\n",
      " - 1s - loss: 0.2172 - acc: 0.9696\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.0108 - acc: 0.9969\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.0038 - acc: 0.9990\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0027 - acc: 0.9992\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9994\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9995\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9996\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 10/32\n",
      " - 0s - loss: 9.9484e-04 - acc: 0.9999\n",
      "Epoch 11/32\n",
      " - 0s - loss: 9.0484e-04 - acc: 0.9999\n",
      "Epoch 12/32\n",
      " - 0s - loss: 8.2920e-04 - acc: 0.9999\n",
      "Epoch 13/32\n",
      " - 0s - loss: 7.4384e-04 - acc: 0.9999\n",
      "Epoch 14/32\n",
      " - 0s - loss: 6.9338e-04 - acc: 0.9999\n",
      "Epoch 15/32\n",
      " - 0s - loss: 6.4245e-04 - acc: 0.9999\n",
      "Epoch 16/32\n",
      " - 0s - loss: 5.9615e-04 - acc: 0.9999\n",
      "Epoch 17/32\n",
      " - 0s - loss: 5.6311e-04 - acc: 0.9999\n",
      "Epoch 18/32\n",
      " - 0s - loss: 5.1595e-04 - acc: 0.9999\n",
      "Epoch 19/32\n",
      " - 0s - loss: 4.9498e-04 - acc: 0.9999\n",
      "Epoch 20/32\n",
      " - 0s - loss: 4.6562e-04 - acc: 0.9999\n",
      "Epoch 21/32\n",
      " - 0s - loss: 4.2944e-04 - acc: 0.9999\n",
      "Epoch 22/32\n",
      " - 0s - loss: 3.9434e-04 - acc: 0.9999\n",
      "Epoch 23/32\n",
      " - 0s - loss: 3.7127e-04 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 3.4666e-04 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 3.2777e-04 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 2.9994e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 2.7547e-04 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 2.5740e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 2.4708e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 2.3547e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 2.1925e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 2.1514e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 2.1648 - acc: 0.7976\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.5806 - acc: 0.9716\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.2741 - acc: 0.9976\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.0188 - acc: 0.9987\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0030 - acc: 0.9990\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0024 - acc: 0.9994\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0022 - acc: 0.9996\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9997\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9996\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0018 - acc: 0.9998\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9998\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9998\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9998\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0015 - acc: 0.9998\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9997\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9997\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0013 - acc: 0.9997\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9998\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9997\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9997\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9997\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0010 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 9.5978e-04 - acc: 0.9997\n",
      "Epoch 24/32\n",
      " - 0s - loss: 8.9655e-04 - acc: 0.9997\n",
      "Epoch 25/32\n",
      " - 0s - loss: 8.4069e-04 - acc: 0.9997\n",
      "Epoch 26/32\n",
      " - 0s - loss: 7.8013e-04 - acc: 0.9997\n",
      "Epoch 27/32\n",
      " - 0s - loss: 7.3071e-04 - acc: 0.9998\n",
      "Epoch 28/32\n",
      " - 0s - loss: 6.9268e-04 - acc: 0.9998\n",
      "Epoch 29/32\n",
      " - 0s - loss: 6.2548e-04 - acc: 0.9998\n",
      "Epoch 30/32\n",
      " - 0s - loss: 5.9897e-04 - acc: 0.9998\n",
      "Epoch 31/32\n",
      " - 0s - loss: 5.2087e-04 - acc: 0.9997\n",
      "Epoch 32/32\n",
      " - 0s - loss: 4.8626e-04 - acc: 0.9997\n",
      "Epoch 1/32\n",
      " - 1s - loss: 7.1274 - acc: 0.5395\n",
      "Epoch 2/32\n",
      " - 0s - loss: 1.3665 - acc: 0.8744\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.1415 - acc: 0.9927\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.1177 - acc: 0.9952\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.0724 - acc: 0.9959\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.0402 - acc: 0.9963\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.0194 - acc: 0.9969\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.0079 - acc: 0.9977\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.0061 - acc: 0.9985\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.0052 - acc: 0.9986\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.0045 - acc: 0.9987\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.0039 - acc: 0.9987\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.0035 - acc: 0.9990\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.0030 - acc: 0.9991\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0028 - acc: 0.9993\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0025 - acc: 0.9994\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0023 - acc: 0.9995\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0020 - acc: 0.9996\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0019 - acc: 0.9998\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0017 - acc: 0.9998\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0016 - acc: 0.9997\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0014 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0012 - acc: 0.9999\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0011 - acc: 0.9999\n",
      "Epoch 25/32\n",
      " - 0s - loss: 9.9757e-04 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 9.2235e-04 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 8.7346e-04 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 8.1856e-04 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 7.6835e-04 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 7.2794e-04 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 6.8645e-04 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 6.4036e-04 - acc: 0.9999\n",
      "Epoch 1/32\n",
      " - 1s - loss: 1.5841 - acc: 0.8406\n",
      "Epoch 2/32\n",
      " - 0s - loss: 0.3627 - acc: 0.9887\n",
      "Epoch 3/32\n",
      " - 0s - loss: 0.3016 - acc: 0.9934\n",
      "Epoch 4/32\n",
      " - 0s - loss: 0.2642 - acc: 0.9965\n",
      "Epoch 5/32\n",
      " - 0s - loss: 0.2373 - acc: 0.9985\n",
      "Epoch 6/32\n",
      " - 0s - loss: 0.2148 - acc: 0.9987\n",
      "Epoch 7/32\n",
      " - 0s - loss: 0.1948 - acc: 0.9989\n",
      "Epoch 8/32\n",
      " - 0s - loss: 0.1768 - acc: 0.9991\n",
      "Epoch 9/32\n",
      " - 0s - loss: 0.1610 - acc: 0.9993\n",
      "Epoch 10/32\n",
      " - 0s - loss: 0.1471 - acc: 0.9993\n",
      "Epoch 11/32\n",
      " - 0s - loss: 0.1347 - acc: 0.9993\n",
      "Epoch 12/32\n",
      " - 0s - loss: 0.1236 - acc: 0.9994\n",
      "Epoch 13/32\n",
      " - 0s - loss: 0.1138 - acc: 0.9994\n",
      "Epoch 14/32\n",
      " - 0s - loss: 0.1050 - acc: 0.9996\n",
      "Epoch 15/32\n",
      " - 0s - loss: 0.0970 - acc: 0.9996\n",
      "Epoch 16/32\n",
      " - 0s - loss: 0.0898 - acc: 0.9996\n",
      "Epoch 17/32\n",
      " - 0s - loss: 0.0833 - acc: 0.9997\n",
      "Epoch 18/32\n",
      " - 0s - loss: 0.0776 - acc: 0.9995\n",
      "Epoch 19/32\n",
      " - 0s - loss: 0.0721 - acc: 0.9997\n",
      "Epoch 20/32\n",
      " - 0s - loss: 0.0674 - acc: 0.9997\n",
      "Epoch 21/32\n",
      " - 0s - loss: 0.0629 - acc: 0.9997\n",
      "Epoch 22/32\n",
      " - 0s - loss: 0.0588 - acc: 0.9998\n",
      "Epoch 23/32\n",
      " - 0s - loss: 0.0550 - acc: 0.9998\n",
      "Epoch 24/32\n",
      " - 0s - loss: 0.0515 - acc: 0.9998\n",
      "Epoch 25/32\n",
      " - 0s - loss: 0.0484 - acc: 0.9999\n",
      "Epoch 26/32\n",
      " - 0s - loss: 0.0455 - acc: 0.9999\n",
      "Epoch 27/32\n",
      " - 0s - loss: 0.0428 - acc: 0.9999\n",
      "Epoch 28/32\n",
      " - 0s - loss: 0.0404 - acc: 0.9999\n",
      "Epoch 29/32\n",
      " - 0s - loss: 0.0381 - acc: 0.9999\n",
      "Epoch 30/32\n",
      " - 0s - loss: 0.0360 - acc: 0.9999\n",
      "Epoch 31/32\n",
      " - 0s - loss: 0.0340 - acc: 0.9999\n",
      "Epoch 32/32\n",
      " - 0s - loss: 0.0322 - acc: 0.9999\n",
      "<class 'keras.wrappers.scikit_learn.KerasClassifier'>\n",
      "[[7884    2]\n",
      " [   5 7109]]\n",
      "total: 15000\n",
      "accuracy: 0.9995333333333334\n",
      "Matthews correlation coefficient: 0.9990642489076085\n",
      "Baseline: 94.67% (15.96%)\n"
     ]
    }
   ],
   "source": [
    "#for i in range(0,10):\n",
    "estimator = KerasClassifier(build_fn=baseline_model, epochs=32, batch_size=200, verbose=2)\n",
    "\n",
    "kfold = KFold(n_splits=10, shuffle=True, random_state=seed)\n",
    "y_pred = cross_val_predict(estimator, X, dummy_y, cv=kfold)\n",
    "results = cross_val_score(estimator, X, dummy_y, cv=kfold)\n",
    "\n",
    "trained_classifier = estimator.fit(X, Y)\n",
    "print(type(estimator))\n",
    "\n",
    "cm = confusion_matrix(Y, y_pred)\n",
    "print(cm)\n",
    "print(\"total: \" + str(cm.sum()))\n",
    "print(\"accuracy: \" + str(np.trace(cm) / cm.sum()))\n",
    "print(\"Matthews correlation coefficient: \" + str(matthews_corrcoef(Y, y_pred)))\n",
    "\n",
    "\n",
    "\n",
    "print(\"Baseline: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))\n",
    "\n",
    "f = open(\"../../Results/discriminatorResults.txt\", \"a+\")\n",
    "f.write(\"TP: %d, FP: %d, FN: %d, TN: %d\\n\" % (cm[0][0], cm[0][1], cm[1][0], cm[1][1]))\n",
    "f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Real neptune attacks:\n",
      "[[ 0  0 17  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0 10  0\n",
      "   1  0  0  0  1  3  0  6  0  0  3  0  0  0  0  0  0  0]\n",
      " [ 0  0 22  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0 34  9\n",
      "   1  0  0  0  8  2  0  6  9  4  1  0  0  0  0  0  0  0]]\n",
      "0 [D loss: 0.382158, acc.: 74.80%] [G loss: 0.631542] [Loss change: 0.632, Loss increases: 0]\n",
      "100 [D loss: 0.622788, acc.: 50.00%] [G loss: 0.401457] [Loss change: -0.003, Loss increases: 0]\n",
      "200 [D loss: 0.662134, acc.: 50.00%] [G loss: 0.358740] [Loss change: -0.006, Loss increases: 0]\n",
      "300 [D loss: 0.694411, acc.: 50.00%] [G loss: 0.328874] [Loss change: -0.016, Loss increases: 1]\n",
      "400 [D loss: 0.710439, acc.: 50.00%] [G loss: 0.320491] [Loss change: 0.007, Loss increases: 1]\n",
      "500 [D loss: 0.719216, acc.: 50.00%] [G loss: 0.320770] [Loss change: 0.010, Loss increases: 0]\n",
      "600 [D loss: 0.743956, acc.: 50.00%] [G loss: 0.305907] [Loss change: 0.000, Loss increases: 0]\n",
      "700 [D loss: 0.758458, acc.: 50.00%] [G loss: 0.295745] [Loss change: -0.001, Loss increases: 0]\n",
      "800 [D loss: 0.768925, acc.: 50.20%] [G loss: 0.299748] [Loss change: 0.004, Loss increases: 0]\n",
      "900 [D loss: 0.754911, acc.: 50.20%] [G loss: 0.301850] [Loss change: 0.008, Loss increases: 0]\n",
      "1000 [D loss: 0.801399, acc.: 50.20%] [G loss: 0.287881] [Loss change: -0.001, Loss increases: 0]\n",
      "1100 [D loss: 0.802424, acc.: 50.20%] [G loss: 0.284828] [Loss change: -0.003, Loss increases: 0]\n",
      "1200 [D loss: 0.788717, acc.: 50.39%] [G loss: 0.294236] [Loss change: 0.001, Loss increases: 1]\n",
      "1300 [D loss: 0.812728, acc.: 50.00%] [G loss: 0.293293] [Loss change: 0.006, Loss increases: 0]\n",
      "1400 [D loss: 0.793049, acc.: 50.20%] [G loss: 0.296297] [Loss change: 0.010, Loss increases: 0]\n",
      "1500 [D loss: 0.809447, acc.: 50.39%] [G loss: 0.297020] [Loss change: 0.002, Loss increases: 2]\n",
      "1600 [D loss: 0.810213, acc.: 50.20%] [G loss: 0.293752] [Loss change: 0.003, Loss increases: 0]\n",
      "1700 [D loss: 0.801299, acc.: 50.00%] [G loss: 0.299191] [Loss change: -0.005, Loss increases: 2]\n",
      "1800 [D loss: 0.775643, acc.: 50.59%] [G loss: 0.306925] [Loss change: 0.011, Loss increases: 1]\n",
      "1900 [D loss: 0.771679, acc.: 50.20%] [G loss: 0.299749] [Loss change: -0.005, Loss increases: 3]\n",
      "2000 [D loss: 0.813127, acc.: 50.78%] [G loss: 0.307908] [Loss change: 0.004, Loss increases: 0]\n",
      "2100 [D loss: 0.792907, acc.: 51.37%] [G loss: 0.312337] [Loss change: 0.009, Loss increases: 0]\n",
      "2200 [D loss: 0.792971, acc.: 50.39%] [G loss: 0.312867] [Loss change: 0.000, Loss increases: 0]\n",
      "2300 [D loss: 0.779905, acc.: 50.78%] [G loss: 0.310626] [Loss change: -0.004, Loss increases: 0]\n",
      "2400 [D loss: 0.774396, acc.: 51.17%] [G loss: 0.317393] [Loss change: 0.003, Loss increases: 0]\n",
      "2500 [D loss: 0.780444, acc.: 50.78%] [G loss: 0.320597] [Loss change: -0.001, Loss increases: 2]\n",
      "2600 [D loss: 0.790578, acc.: 50.00%] [G loss: 0.312614] [Loss change: -0.005, Loss increases: 0]\n",
      "2700 [D loss: 0.776593, acc.: 50.39%] [G loss: 0.324110] [Loss change: 0.004, Loss increases: 0]\n",
      "2800 [D loss: 0.762642, acc.: 50.20%] [G loss: 0.324073] [Loss change: 0.010, Loss increases: 0]\n",
      "2900 [D loss: 0.774046, acc.: 49.61%] [G loss: 0.316438] [Loss change: 0.001, Loss increases: 0]\n",
      "3000 [D loss: 0.797439, acc.: 50.00%] [G loss: 0.304471] [Loss change: -0.004, Loss increases: 0]\n",
      "Stoping on iteration:  3051\n",
      "Generated Neptune attacks: \n",
      "[[-0. -0. 32. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0.\n",
      "  -0. -0. -0. -0. 35. 12.  1. -0. -0. -0. 17.  5. -0. 17. 10. -0.  5. -0.\n",
      "  -0. -0. -0. -0. -0.]\n",
      " [-0. -0. 19. -0. -0. -0.  1.  0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0.\n",
      "  -0. -0. -0. -0. 74.  3. -0. -0. -0. -0.  3.  2. -0. -0. -0. -0.  3. -0.\n",
      "  -0. -0. -0. -0. -0.]]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "Attack type: neptune.     number predicted:  256\n",
      "\n",
      "[[256]]\n",
      "total: 256\n",
      "accuracy: 1.0\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../../Results/../../Results/GANresultsNeptune739075iter0.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-0340ac8394cb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0mresult_filename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"../../Results/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mresult_filename\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"w\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m             \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavetxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfmt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"%.0f\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../Results/../../Results/GANresultsNeptune739075iter0.txt'"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "f = open(\"GeneratorHypersAbove50percentAccuracy.txt\", \"w\")\n",
    "f.write(\"\"\"\"\"\" Hidden layer counts for Generator model that resulted in over 50% generated attacks labeled correctly:\n",
    "    ------------------------------------------------------------------------------------------------\n",
    "    \"\"\"\"\"\")\n",
    "f.close()\n",
    "\"\"\"\n",
    "\n",
    "while(1):\n",
    "    # generate random numbers for the hidden layer sizes of our generator\n",
    "    gen_hidden1 = np.random.randint(1, 101)\n",
    "    gen_hidden2 = np.random.randint(1, 101)\n",
    "    gen_hidden3 = np.random.randint(1, 101)\n",
    "    \n",
    "    i = 0\n",
    "    \n",
    "    \n",
    "    # train 5 times on each setup, in case we get unlucky initalization on an otherwise good setup\n",
    "    while i < 5:\n",
    "        # create a unique filename in case we want to store the results (good accuracy)\n",
    "        result_filename = \"../../Results/GANresultsNeptune%.0f%.0f%.0fiter%.0f.txt\" % (gen_hidden1, gen_hidden2, gen_hidden3, i)\n",
    "\n",
    "        trainGAN(gen_hidden1, gen_hidden2, gen_hidden3)\n",
    "        \n",
    "        # load generate attacks from file\n",
    "        results = np.loadtxt(\"../../Results/GANresultsNeptune.txt\")\n",
    "\n",
    "        # predict attack lables (as encoded integers)\n",
    "        y_pred = estimator.predict(results)\n",
    "        print(y_pred)\n",
    "\n",
    "        # create appropriate labels for our generated neptune attacks\n",
    "        neptune_labels = np.full((len(results),), neptune_index[0])\n",
    "\n",
    "        # convert integer labels back to string, get all unique strings and their count\n",
    "        predicted_as_label = attack_labels[y_pred]\n",
    "        unique_labels = np.unique(predicted_as_label)\n",
    "\n",
    "        for label in unique_labels:\n",
    "            print(\"Attack type: %s     number predicted:  %.0f\" % (label, len(np.where(predicted_as_label == label)[0])))\n",
    "    \n",
    "        print()\n",
    "        # create a confusion matrix of the results\n",
    "        cm = confusion_matrix(neptune_labels, y_pred)\n",
    "        \n",
    "        accuracy = np.trace(cm) / cm.sum()\n",
    "        print(cm)\n",
    "        print(\"total: \" + str(cm.sum()))\n",
    "        print(\"accuracy: \" + str(accuracy))\n",
    "        \n",
    "        if accuracy > .50:\n",
    "            f = open(\"../../Results/GeneratorHypersAbove50percentAccuracyNeptune.txt\", \"a\")\n",
    "            f.write(\"\"\"\n",
    "            \n",
    "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
    "Accuracy: %.3f\n",
    "Generator hidden layer 1 size: %.0f\n",
    "Generator hidden layer 2 size: %.0f\n",
    "Generator hidden layer 3 size: %.0f\n",
    "Iteration %.0f\n",
    "Result file name: %s\n",
    "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\"\"\" % (accuracy, gen_hidden1, gen_hidden2, gen_hidden3, i, result_filename))\n",
    "            f.close()\n",
    "            result_filename = \"../../Results/\" + result_filename\n",
    "            \n",
    "            f = open(result_filename, \"w\")\n",
    "            f.close()\n",
    "            np.savetxt(result_filename, results, fmt=\"%.0f\")\n",
    "        \n",
    "        i = i + 1\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
